package made.ql;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.util.Random;
import java.util.Scanner;

import made.ann.Neuron;
import made.util.Config;
import made.util.Logger;
import made.util.Helpers;
import made.util.Config.DEBUG_LEVEL;

public class QLearning implements Config
{
    // --- variables
    protected int _states;
    protected int _actions;
    protected int _moves;
    protected float _deltaQAccumulated;
    protected float _currentEpsilon;
    protected float _currentAlpha;
    protected STATE _previousState;
    protected ACTION _previousAction;

    // --- objects
    protected float[][] _table;
    protected Random _randomGenerator;

    public QLearning(int states, int actions)
    {
        _states = states;
        _actions = actions;
        _table = new float[_states][_actions];
        _randomGenerator = new Random();
        _currentEpsilon = EPSILON;
        _currentAlpha = ALPHA;
        init();
    }

    public void init()
    {
        _deltaQAccumulated = 0.0f;
        _previousState = STATE.UNDEFINED;
        _previousAction = ACTION.UNDEFINED;
        _moves = 0;
        
        if (MPM_MODE == MODE.CONFIG)
            initConfig();
    }
    
    protected void initConfig()
    {
        _table[0][0] = -74.25389f;
        _table[0][1] = -193.7839f;
        _table[0][2] = -18.737461f;
        _table[1][0] = 8.423931f;
        _table[1][1] = 5.132525f;
        _table[1][2] = 6.828638f;
        _table[2][0] = 425.34988f;
        _table[2][1] = 525.614f;
        _table[2][2] = 317.51196f;
        _table[3][0] = 11.660522f;
        _table[3][1] = 2.9988334f;
        _table[3][2] = 2.246577f;
        
    }

    public ACTION getNextAction(STATE state)
    {
        if (MPM_DEBUG_LEVEL.compareTo(DEBUG_LEVEL.NORMAL) > 0)
            Logger.logLine("* Get next action *\n");

        _previousState = state;

        if (_randomGenerator.nextFloat() < _currentEpsilon)
        {
            _previousAction = explore(_previousState);
        } else
        {
            _previousAction = getBestAction(_previousState);
        }

        return _previousAction;
    }

    public ACTION getBestAction(STATE state)
    {
        if (MPM_DEBUG_LEVEL.compareTo(DEBUG_LEVEL.NORMAL) > 0)
            Logger.logLine("* Get the best action *");

        int bestAction = -1;
        float maxQ = Float.NEGATIVE_INFINITY;

        for (int i = 0; i < _actions; i++)
        {
            float nextQ = _table[state.ordinal()][i];

            if (nextQ > maxQ)
            {
                bestAction = i;
                maxQ = nextQ;
            }
        }

        if (MPM_DEBUG_LEVEL.compareTo(DEBUG_LEVEL.NORMAL) > 0)
        {
            Logger.logLine("Best action from state " + state.toString() + " is " + Helpers.toActionEnum(bestAction).toString());
            Logger.logLine();
        }

        return Helpers.toActionEnum(bestAction);
    }

    public float updateQvalue(STATE state, float reward)
    {
        if (MPM_DEBUG_LEVEL.compareTo(DEBUG_LEVEL.NORMAL) > 0)
            Logger.logLine("* Update Q table *");

        decreaseAlpha();
        decreaseEpsilon();
        _moves++;

        // Don't update if this is the first run
        if (_previousState != STATE.UNDEFINED && _previousAction != ACTION.UNDEFINED)
        {
            float oldQ = _table[_previousState.ordinal()][_previousAction.ordinal()];
            float nextQ = _table[state.ordinal()][getBestAction(state).ordinal()];
            float deltaQ = _currentAlpha * (reward + GAMMA * nextQ - oldQ);
            _table[_previousState.ordinal()][_previousAction.ordinal()] = oldQ + deltaQ;
            _deltaQAccumulated += deltaQ;

            if (MPM_DEBUG_LEVEL.compareTo(DEBUG_LEVEL.NORMAL) > 0)
            {
                Logger.logLine("Updated [" + _previousState.toString() + "][" + _previousAction.toString() + "] from " + oldQ + " to "
                        + _table[_previousState.ordinal()][_previousAction.ordinal()] + "\n");
            }
            
            return deltaQ;
        } else
        {
            return 0.0f;
        }
    }

    protected void decreaseAlpha()
    {
        _currentAlpha -= _currentAlpha * ALPHA_DECREASE;

        if (MPM_DEBUG_LEVEL.compareTo(DEBUG_LEVEL.NORMAL) > 0)
            Logger.logLine("Alpha decreased to " + _currentAlpha);
    }

    protected void decreaseEpsilon()
    {
        _currentEpsilon -= _currentEpsilon * EPSILON_DECREASE;

        if (MPM_DEBUG_LEVEL.compareTo(DEBUG_LEVEL.NORMAL) > 0)
            Logger.logLine("Epsilon decreased to " + _currentEpsilon);
    }
    
    protected void printDeltaQAverage()
    {
        Logger.log(" " + _deltaQAccumulated / _moves);
    }

    public boolean hasConverged(int moves, float threshold)
    {
        boolean converged = false;
        float getDeltaQAverage = _deltaQAccumulated / moves;
//        Logger.logLine("_deltaQAccumulated: " + _deltaQAccumulated);
//        Logger.logLine("nextdeltaQAccumulated: " + nextdeltaQAccumulated);

        if (Math.abs(getDeltaQAverage) < threshold)
        {
            converged = true;
            saveTable();
        }

        return converged;
    }

    public void printTable()
    {
        Logger.logLine("------------------------------------------------------------------------------------");
        Logger.logLine("--------------------------- Current Q table ----------------------------------------");
        Logger.logLine("------------------------------------------------------------------------------------");

        Logger.logMethod();
        // Logger.log("\t\t");
        System.out.printf("%15s", "");

        for (int i = 0; i < _actions; i++)
        {
            System.out.printf(" %22s", Helpers.toActionEnum(i).toString());
        }

        Logger.logLine();

        for (int i = 0; i < _states; i++)
        {
            Logger.logMethod();
            System.out.printf("%15s", Helpers.toStateEnum(i).toString());

            for (int j = 0; j < _actions; j++)
            {
                System.out.printf("%20.2f", _table[i][j]);
            }

            Logger.logLine();

            if (i == _states - 1)
                Logger.logLine("------------------------------------------------------------------------------------");
        }
    }

    protected void loadTable(String filename)
    {
        // if (MPM_DEBUG_LEVEL == DEBUG_LEVEL.VERBOSE)
        Logger.logLine("* Initializing Q table from file *\n");

        File file = new File(filename);
        String nextLine = "";
        Scanner fileScanner;

        try
        {
            fileScanner = new Scanner(new FileReader(file));

            try
            {
                for (int i = 1; i < _states; i++)
                {
                    // Logger.logLine("i: " + i + " FileScanner hasNext: " +
                    // fileScanner.hasNext());
                    nextLine = fileScanner.nextLine();
                    Scanner lineScanner = new Scanner(nextLine);
                    lineScanner.useDelimiter(" ");

                    for (int j = 0; j < _actions; j++)
                    {
                        _table[i][j] = Float.parseFloat(lineScanner.next());

                        if (MPM_DEBUG_LEVEL == DEBUG_LEVEL.VERBOSE)
                            Logger.logLine("_table[" + i + "][" + j + "] = " + _table[i][j]);
                    }

                    lineScanner.close();
                }
            } catch (Exception e)
            {
                Logger.logLine("An error ocurred while reading the file '" + filename + "'");
                Logger.logLine("Exiting...");
                System.exit(0);
            } finally
            {
                fileScanner.close();
            }
        } catch (FileNotFoundException e)
        {
            Logger.logLine("Could not find the file '" + filename + "'");
            Logger.logLine("Exiting...");
            System.exit(0);
        }
    }

    protected void saveTable()
    {
        if (MPM_DEBUG_LEVEL == DEBUG_LEVEL.VERBOSE)
            Logger.logLine("* Initializing neural network from file *\n");

        BufferedWriter bufferedWriter = null;

        try
        {
            bufferedWriter = new BufferedWriter(new FileWriter(filepath + "ql_config.txt", false));

            for (int i = 0; i < _states; i++)
            {
                String nextLine = "";

                for (int j = 0; j < _actions; j++)
                {
                    nextLine += _table[i][j];
                    nextLine += " ";
                }

                if (MPM_DEBUG_LEVEL == DEBUG_LEVEL.VERBOSE)
                    Logger.logLine(nextLine);

                bufferedWriter.write(nextLine);
                bufferedWriter.newLine();
            }
        } catch (FileNotFoundException ex)
        {
            ex.printStackTrace();
        } catch (IOException ex)
        {
            ex.printStackTrace();
        } finally
        {
            // Close the BufferedWriter
            try
            {
                if (bufferedWriter != null)
                {
                    bufferedWriter.flush();
                    bufferedWriter.close();
                }
            } catch (IOException ex)
            {
                ex.printStackTrace();
            }
        }
    }

    protected ACTION explore(STATE state)
    {
        if (MPM_DEBUG_LEVEL.compareTo(DEBUG_LEVEL.NORMAL) > 0)
            Logger.logLine("* Explore random action *\n");

        ACTION nextAction = ACTION.UNDEFINED;
        ACTION bestAction = getBestAction(state);

        while (true)
        {
            nextAction = Helpers.toActionEnum(_randomGenerator.nextInt(_actions));

            if (nextAction != bestAction)
                break;
        }

        if (MPM_DEBUG_LEVEL.compareTo(DEBUG_LEVEL.NORMAL) > 0)
        {
            Logger.logLine("Chosen action from state " + state.toString() + " is " + nextAction.toString());
            Logger.logLine();
        }

        return nextAction;
    }
};